

BOT_NAME = "web_crawler"
LOG_LEVEL = 'DEBUG'
SPIDER_MODULES = ["web_crawler.spiders"]
NEWSPIDER_MODULE = "web_crawler.spiders"

ADDONS = {}

ROBOTSTXT_OBEY = False
CONCURRENT_REQUESTS_PER_DOMAIN = 1
DOWNLOAD_DELAY = 1
CONCURRENT_REQUESTS = 8
COOKIES_ENABLED = True

FEED_EXPORT_ENCODING = "utf-8"
# MySQL数据库配置
MYSQL_HOST = 'localhost'      # 数据库服务器地址
MYSQL_PORT = 3306             # 数据库端口，默认为3306
MYSQL_USER = 'root'  # 数据库用户名
MYSQL_PASSWORD = '111111'  # 数据库密码
MYSQL_DATABASE = 'xueqiu_data'  # 数据库名称
MYSQL_CHARSET = 'utf8mb4'     # 数据库字符集

ITEM_PIPELINES = {
    'web_crawler.pipeline.debug.WebCrawlerPipeline':300
}

SPIDER_MIDDLEWARES = {
    'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,  # 临时关闭可能干扰的中间件
}